{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from os import listdir\n",
    "from pickle import dump\n",
    "from keras.applications.vgg16 import VGG16\n",
    "from keras.preprocessing.image import load_img\n",
    "from keras.preprocessing.image import img_to_array\n",
    "from keras.applications.vgg16 import preprocess_input\n",
    "from keras.layers import Input, Reshape, Concatenate, Activation, Dense\n",
    "import keras\n",
    "import numpy as np\n",
    "import string\n",
    "# from progressbar import progressbar\n",
    "from keras.models import Model\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_4 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "=================================================================\n",
      "Total params: 134,260,544\n",
      "Trainable params: 134,260,544\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import SVG\n",
    "from keras.utils.vis_utils import model_to_dot\n",
    "\n",
    "# load model\n",
    "model = VGG16()\n",
    "model.layers.pop()\n",
    "# extract final 49x512 conv layer for context vectors\n",
    "final_conv = Reshape([49,512])(model.layers[-4].output)\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"1754pt\" viewBox=\"0.00 0.00 382.01 1754.00\" width=\"382pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 1750)\">\n",
       "<title>G</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-1750 378.013,-1750 378.013,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 103551592992 -->\n",
       "<g class=\"node\" id=\"node1\"><title>103551592992</title>\n",
       "<polygon fill=\"none\" points=\"32.1665,-1701.5 32.1665,-1745.5 341.846,-1745.5 341.846,-1701.5 32.1665,-1701.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-1719.3\">input_4: InputLayer</text>\n",
       "<polyline fill=\"none\" points=\"160.529,-1701.5 160.529,-1745.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"188.363\" y=\"-1730.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"160.529,-1723.5 216.198,-1723.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"188.363\" y=\"-1708.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"216.198,-1701.5 216.198,-1745.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"279.022\" y=\"-1730.3\">(None, 224, 224, 3)</text>\n",
       "<polyline fill=\"none\" points=\"216.198,-1723.5 341.846,-1723.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"279.022\" y=\"-1708.3\">(None, 224, 224, 3)</text>\n",
       "</g>\n",
       "<!-- 103552155264 -->\n",
       "<g class=\"node\" id=\"node2\"><title>103552155264</title>\n",
       "<polygon fill=\"none\" points=\"17.77,-1620.5 17.77,-1664.5 356.243,-1664.5 356.243,-1620.5 17.77,-1620.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"92.8477\" y=\"-1638.3\">block1_conv1: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"167.925,-1620.5 167.925,-1664.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"195.76\" y=\"-1649.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"167.925,-1642.5 223.594,-1642.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"195.76\" y=\"-1627.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"223.594,-1620.5 223.594,-1664.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1649.3\">(None, 224, 224, 3)</text>\n",
       "<polyline fill=\"none\" points=\"223.594,-1642.5 356.243,-1642.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1627.3\">(None, 224, 224, 64)</text>\n",
       "</g>\n",
       "<!-- 103551592992&#45;&gt;103552155264 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>103551592992-&gt;103552155264</title>\n",
       "<path d=\"M187.006,-1701.33C187.006,-1693.18 187.006,-1683.7 187.006,-1674.8\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-1674.73 187.006,-1664.73 183.506,-1674.73 190.506,-1674.73\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 104042176184 -->\n",
       "<g class=\"node\" id=\"node3\"><title>104042176184</title>\n",
       "<polygon fill=\"none\" points=\"17.77,-1539.5 17.77,-1583.5 356.243,-1583.5 356.243,-1539.5 17.77,-1539.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"92.8477\" y=\"-1557.3\">block1_conv2: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"167.925,-1539.5 167.925,-1583.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"195.76\" y=\"-1568.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"167.925,-1561.5 223.594,-1561.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"195.76\" y=\"-1546.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"223.594,-1539.5 223.594,-1583.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1568.3\">(None, 224, 224, 64)</text>\n",
       "<polyline fill=\"none\" points=\"223.594,-1561.5 356.243,-1561.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1546.3\">(None, 224, 224, 64)</text>\n",
       "</g>\n",
       "<!-- 103552155264&#45;&gt;104042176184 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>103552155264-&gt;104042176184</title>\n",
       "<path d=\"M187.006,-1620.33C187.006,-1612.18 187.006,-1602.7 187.006,-1593.8\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-1593.73 187.006,-1583.73 183.506,-1593.73 190.506,-1593.73\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 104042175736 -->\n",
       "<g class=\"node\" id=\"node4\"><title>104042175736</title>\n",
       "<polygon fill=\"none\" points=\"2.9873,-1458.5 2.9873,-1502.5 371.025,-1502.5 371.025,-1458.5 2.9873,-1458.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"92.8477\" y=\"-1476.3\">block1_pool: MaxPooling2D</text>\n",
       "<polyline fill=\"none\" points=\"182.708,-1458.5 182.708,-1502.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"210.542\" y=\"-1487.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"182.708,-1480.5 238.377,-1480.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"210.542\" y=\"-1465.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"238.377,-1458.5 238.377,-1502.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"304.701\" y=\"-1487.3\">(None, 224, 224, 64)</text>\n",
       "<polyline fill=\"none\" points=\"238.377,-1480.5 371.025,-1480.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"304.688\" y=\"-1465.3\">(None, 112, 112, 64)</text>\n",
       "</g>\n",
       "<!-- 104042176184&#45;&gt;104042175736 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>104042176184-&gt;104042175736</title>\n",
       "<path d=\"M187.006,-1539.33C187.006,-1531.18 187.006,-1521.7 187.006,-1512.8\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-1512.73 187.006,-1502.73 183.506,-1512.73 190.506,-1512.73\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103554164664 -->\n",
       "<g class=\"node\" id=\"node5\"><title>103554164664</title>\n",
       "<polygon fill=\"none\" points=\"14.7827,-1377.5 14.7827,-1421.5 359.23,-1421.5 359.23,-1377.5 14.7827,-1377.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"89.8604\" y=\"-1395.3\">block2_conv1: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"164.938,-1377.5 164.938,-1421.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"192.772\" y=\"-1406.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"164.938,-1399.5 220.607,-1399.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"192.772\" y=\"-1384.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"220.607,-1377.5 220.607,-1421.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1406.3\">(None, 112, 112, 64)</text>\n",
       "<polyline fill=\"none\" points=\"220.607,-1399.5 359.23,-1399.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1384.3\">(None, 112, 112, 128)</text>\n",
       "</g>\n",
       "<!-- 104042175736&#45;&gt;103554164664 -->\n",
       "<g class=\"edge\" id=\"edge4\"><title>104042175736-&gt;103554164664</title>\n",
       "<path d=\"M187.006,-1458.33C187.006,-1450.18 187.006,-1440.7 187.006,-1431.8\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-1431.73 187.006,-1421.73 183.506,-1431.73 190.506,-1431.73\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103554166176 -->\n",
       "<g class=\"node\" id=\"node6\"><title>103554166176</title>\n",
       "<polygon fill=\"none\" points=\"14.7827,-1296.5 14.7827,-1340.5 359.23,-1340.5 359.23,-1296.5 14.7827,-1296.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"89.8604\" y=\"-1314.3\">block2_conv2: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"164.938,-1296.5 164.938,-1340.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"192.772\" y=\"-1325.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"164.938,-1318.5 220.607,-1318.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"192.772\" y=\"-1303.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"220.607,-1296.5 220.607,-1340.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1325.3\">(None, 112, 112, 128)</text>\n",
       "<polyline fill=\"none\" points=\"220.607,-1318.5 359.23,-1318.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1303.3\">(None, 112, 112, 128)</text>\n",
       "</g>\n",
       "<!-- 103554164664&#45;&gt;103554166176 -->\n",
       "<g class=\"edge\" id=\"edge5\"><title>103554164664-&gt;103554166176</title>\n",
       "<path d=\"M187.006,-1377.33C187.006,-1369.18 187.006,-1359.7 187.006,-1350.8\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-1350.73 187.006,-1340.73 183.506,-1350.73 190.506,-1350.73\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103555205888 -->\n",
       "<g class=\"node\" id=\"node7\"><title>103555205888</title>\n",
       "<polygon fill=\"none\" points=\"0,-1215.5 0,-1259.5 374.013,-1259.5 374.013,-1215.5 0,-1215.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"89.8604\" y=\"-1233.3\">block2_pool: MaxPooling2D</text>\n",
       "<polyline fill=\"none\" points=\"179.721,-1215.5 179.721,-1259.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"207.555\" y=\"-1244.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"179.721,-1237.5 235.39,-1237.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"207.555\" y=\"-1222.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"235.39,-1215.5 235.39,-1259.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"304.701\" y=\"-1244.3\">(None, 112, 112, 128)</text>\n",
       "<polyline fill=\"none\" points=\"235.39,-1237.5 374.013,-1237.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"304.214\" y=\"-1222.3\">(None, 56, 56, 128)</text>\n",
       "</g>\n",
       "<!-- 103554166176&#45;&gt;103555205888 -->\n",
       "<g class=\"edge\" id=\"edge6\"><title>103554166176-&gt;103555205888</title>\n",
       "<path d=\"M187.006,-1296.33C187.006,-1288.18 187.006,-1278.7 187.006,-1269.8\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-1269.73 187.006,-1259.73 183.506,-1269.73 190.506,-1269.73\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103555348352 -->\n",
       "<g class=\"node\" id=\"node8\"><title>103555348352</title>\n",
       "<polygon fill=\"none\" points=\"21.27,-1134.5 21.27,-1178.5 352.743,-1178.5 352.743,-1134.5 21.27,-1134.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-1152.3\">block3_conv1: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-1134.5 171.425,-1178.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-1163.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-1156.5 227.094,-1156.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-1141.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-1134.5 227.094,-1178.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1163.3\">(None, 56, 56, 128)</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-1156.5 352.743,-1156.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1141.3\">(None, 56, 56, 256)</text>\n",
       "</g>\n",
       "<!-- 103555205888&#45;&gt;103555348352 -->\n",
       "<g class=\"edge\" id=\"edge7\"><title>103555205888-&gt;103555348352</title>\n",
       "<path d=\"M187.006,-1215.33C187.006,-1207.18 187.006,-1197.7 187.006,-1188.8\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-1188.73 187.006,-1178.73 183.506,-1188.73 190.506,-1188.73\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103555350312 -->\n",
       "<g class=\"node\" id=\"node9\"><title>103555350312</title>\n",
       "<polygon fill=\"none\" points=\"21.27,-1053.5 21.27,-1097.5 352.743,-1097.5 352.743,-1053.5 21.27,-1053.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-1071.3\">block3_conv2: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-1053.5 171.425,-1097.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-1082.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-1075.5 227.094,-1075.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-1060.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-1053.5 227.094,-1097.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1082.3\">(None, 56, 56, 256)</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-1075.5 352.743,-1075.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1060.3\">(None, 56, 56, 256)</text>\n",
       "</g>\n",
       "<!-- 103555348352&#45;&gt;103555350312 -->\n",
       "<g class=\"edge\" id=\"edge8\"><title>103555348352-&gt;103555350312</title>\n",
       "<path d=\"M187.006,-1134.33C187.006,-1126.18 187.006,-1116.7 187.006,-1107.8\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-1107.73 187.006,-1097.73 183.506,-1107.73 190.506,-1107.73\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103555550848 -->\n",
       "<g class=\"node\" id=\"node10\"><title>103555550848</title>\n",
       "<polygon fill=\"none\" points=\"21.27,-972.5 21.27,-1016.5 352.743,-1016.5 352.743,-972.5 21.27,-972.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-990.3\">block3_conv3: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-972.5 171.425,-1016.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-1001.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-994.5 227.094,-994.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-979.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-972.5 227.094,-1016.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-1001.3\">(None, 56, 56, 256)</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-994.5 352.743,-994.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-979.3\">(None, 56, 56, 256)</text>\n",
       "</g>\n",
       "<!-- 103555350312&#45;&gt;103555550848 -->\n",
       "<g class=\"edge\" id=\"edge9\"><title>103555350312-&gt;103555550848</title>\n",
       "<path d=\"M187.006,-1053.33C187.006,-1045.18 187.006,-1035.7 187.006,-1026.8\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-1026.73 187.006,-1016.73 183.506,-1026.73 190.506,-1026.73\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103544888848 -->\n",
       "<g class=\"node\" id=\"node11\"><title>103544888848</title>\n",
       "<polygon fill=\"none\" points=\"6.4873,-891.5 6.4873,-935.5 367.525,-935.5 367.525,-891.5 6.4873,-891.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-909.3\">block3_pool: MaxPooling2D</text>\n",
       "<polyline fill=\"none\" points=\"186.208,-891.5 186.208,-935.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"214.042\" y=\"-920.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"186.208,-913.5 241.877,-913.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"214.042\" y=\"-898.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"241.877,-891.5 241.877,-935.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"304.701\" y=\"-920.3\">(None, 56, 56, 256)</text>\n",
       "<polyline fill=\"none\" points=\"241.877,-913.5 367.525,-913.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"304.701\" y=\"-898.3\">(None, 28, 28, 256)</text>\n",
       "</g>\n",
       "<!-- 103555550848&#45;&gt;103544888848 -->\n",
       "<g class=\"edge\" id=\"edge10\"><title>103555550848-&gt;103544888848</title>\n",
       "<path d=\"M187.006,-972.329C187.006,-964.183 187.006,-954.699 187.006,-945.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-945.729 187.006,-935.729 183.506,-945.729 190.506,-945.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103549706704 -->\n",
       "<g class=\"node\" id=\"node12\"><title>103549706704</title>\n",
       "<polygon fill=\"none\" points=\"21.27,-810.5 21.27,-854.5 352.743,-854.5 352.743,-810.5 21.27,-810.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-828.3\">block4_conv1: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-810.5 171.425,-854.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-839.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-832.5 227.094,-832.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-817.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-810.5 227.094,-854.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-839.3\">(None, 28, 28, 256)</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-832.5 352.743,-832.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-817.3\">(None, 28, 28, 512)</text>\n",
       "</g>\n",
       "<!-- 103544888848&#45;&gt;103549706704 -->\n",
       "<g class=\"edge\" id=\"edge11\"><title>103544888848-&gt;103549706704</title>\n",
       "<path d=\"M187.006,-891.329C187.006,-883.183 187.006,-873.699 187.006,-864.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-864.729 187.006,-854.729 183.506,-864.729 190.506,-864.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103549707824 -->\n",
       "<g class=\"node\" id=\"node13\"><title>103549707824</title>\n",
       "<polygon fill=\"none\" points=\"21.27,-729.5 21.27,-773.5 352.743,-773.5 352.743,-729.5 21.27,-729.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-747.3\">block4_conv2: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-729.5 171.425,-773.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-758.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-751.5 227.094,-751.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-736.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-729.5 227.094,-773.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-758.3\">(None, 28, 28, 512)</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-751.5 352.743,-751.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-736.3\">(None, 28, 28, 512)</text>\n",
       "</g>\n",
       "<!-- 103549706704&#45;&gt;103549707824 -->\n",
       "<g class=\"edge\" id=\"edge12\"><title>103549706704-&gt;103549707824</title>\n",
       "<path d=\"M187.006,-810.329C187.006,-802.183 187.006,-792.699 187.006,-783.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-783.729 187.006,-773.729 183.506,-783.729 190.506,-783.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103553098696 -->\n",
       "<g class=\"node\" id=\"node14\"><title>103553098696</title>\n",
       "<polygon fill=\"none\" points=\"21.27,-648.5 21.27,-692.5 352.743,-692.5 352.743,-648.5 21.27,-648.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-666.3\">block4_conv3: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-648.5 171.425,-692.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-677.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-670.5 227.094,-670.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-655.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-648.5 227.094,-692.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-677.3\">(None, 28, 28, 512)</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-670.5 352.743,-670.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-655.3\">(None, 28, 28, 512)</text>\n",
       "</g>\n",
       "<!-- 103549707824&#45;&gt;103553098696 -->\n",
       "<g class=\"edge\" id=\"edge13\"><title>103549707824-&gt;103553098696</title>\n",
       "<path d=\"M187.006,-729.329C187.006,-721.183 187.006,-711.699 187.006,-702.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-702.729 187.006,-692.729 183.506,-702.729 190.506,-702.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103553245816 -->\n",
       "<g class=\"node\" id=\"node15\"><title>103553245816</title>\n",
       "<polygon fill=\"none\" points=\"6.4873,-567.5 6.4873,-611.5 367.525,-611.5 367.525,-567.5 6.4873,-567.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-585.3\">block4_pool: MaxPooling2D</text>\n",
       "<polyline fill=\"none\" points=\"186.208,-567.5 186.208,-611.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"214.042\" y=\"-596.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"186.208,-589.5 241.877,-589.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"214.042\" y=\"-574.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"241.877,-567.5 241.877,-611.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"304.701\" y=\"-596.3\">(None, 28, 28, 512)</text>\n",
       "<polyline fill=\"none\" points=\"241.877,-589.5 367.525,-589.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"304.701\" y=\"-574.3\">(None, 14, 14, 512)</text>\n",
       "</g>\n",
       "<!-- 103553098696&#45;&gt;103553245816 -->\n",
       "<g class=\"edge\" id=\"edge14\"><title>103553098696-&gt;103553245816</title>\n",
       "<path d=\"M187.006,-648.329C187.006,-640.183 187.006,-630.699 187.006,-621.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-621.729 187.006,-611.729 183.506,-621.729 190.506,-621.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103553110704 -->\n",
       "<g class=\"node\" id=\"node16\"><title>103553110704</title>\n",
       "<polygon fill=\"none\" points=\"21.27,-486.5 21.27,-530.5 352.743,-530.5 352.743,-486.5 21.27,-486.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-504.3\">block5_conv1: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-486.5 171.425,-530.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-515.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-508.5 227.094,-508.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-493.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-486.5 227.094,-530.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-515.3\">(None, 14, 14, 512)</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-508.5 352.743,-508.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-493.3\">(None, 14, 14, 512)</text>\n",
       "</g>\n",
       "<!-- 103553245816&#45;&gt;103553110704 -->\n",
       "<g class=\"edge\" id=\"edge15\"><title>103553245816-&gt;103553110704</title>\n",
       "<path d=\"M187.006,-567.329C187.006,-559.183 187.006,-549.699 187.006,-540.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-540.729 187.006,-530.729 183.506,-540.729 190.506,-540.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103551826688 -->\n",
       "<g class=\"node\" id=\"node17\"><title>103551826688</title>\n",
       "<polygon fill=\"none\" points=\"21.27,-405.5 21.27,-449.5 352.743,-449.5 352.743,-405.5 21.27,-405.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-423.3\">block5_conv2: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-405.5 171.425,-449.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-434.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-427.5 227.094,-427.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-412.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-405.5 227.094,-449.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-434.3\">(None, 14, 14, 512)</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-427.5 352.743,-427.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-412.3\">(None, 14, 14, 512)</text>\n",
       "</g>\n",
       "<!-- 103553110704&#45;&gt;103551826688 -->\n",
       "<g class=\"edge\" id=\"edge16\"><title>103553110704-&gt;103551826688</title>\n",
       "<path d=\"M187.006,-486.329C187.006,-478.183 187.006,-468.699 187.006,-459.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-459.729 187.006,-449.729 183.506,-459.729 190.506,-459.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103549390408 -->\n",
       "<g class=\"node\" id=\"node18\"><title>103549390408</title>\n",
       "<polygon fill=\"none\" points=\"21.27,-324.5 21.27,-368.5 352.743,-368.5 352.743,-324.5 21.27,-324.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-342.3\">block5_conv3: Conv2D</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-324.5 171.425,-368.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-353.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"171.425,-346.5 227.094,-346.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"199.26\" y=\"-331.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-324.5 227.094,-368.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-353.3\">(None, 14, 14, 512)</text>\n",
       "<polyline fill=\"none\" points=\"227.094,-346.5 352.743,-346.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"289.918\" y=\"-331.3\">(None, 14, 14, 512)</text>\n",
       "</g>\n",
       "<!-- 103551826688&#45;&gt;103549390408 -->\n",
       "<g class=\"edge\" id=\"edge17\"><title>103551826688-&gt;103549390408</title>\n",
       "<path d=\"M187.006,-405.329C187.006,-397.183 187.006,-387.699 187.006,-378.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-378.729 187.006,-368.729 183.506,-378.729 190.506,-378.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103551811880 -->\n",
       "<g class=\"node\" id=\"node19\"><title>103551811880</title>\n",
       "<polygon fill=\"none\" points=\"6.4873,-243.5 6.4873,-287.5 367.525,-287.5 367.525,-243.5 6.4873,-243.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"96.3477\" y=\"-261.3\">block5_pool: MaxPooling2D</text>\n",
       "<polyline fill=\"none\" points=\"186.208,-243.5 186.208,-287.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"214.042\" y=\"-272.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"186.208,-265.5 241.877,-265.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"214.042\" y=\"-250.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"241.877,-243.5 241.877,-287.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"304.701\" y=\"-272.3\">(None, 14, 14, 512)</text>\n",
       "<polyline fill=\"none\" points=\"241.877,-265.5 367.525,-265.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"304.701\" y=\"-250.3\">(None, 7, 7, 512)</text>\n",
       "</g>\n",
       "<!-- 103549390408&#45;&gt;103551811880 -->\n",
       "<g class=\"edge\" id=\"edge18\"><title>103549390408-&gt;103551811880</title>\n",
       "<path d=\"M187.006,-324.329C187.006,-316.183 187.006,-306.699 187.006,-297.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-297.729 187.006,-287.729 183.506,-297.729 190.506,-297.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103551754368 -->\n",
       "<g class=\"node\" id=\"node20\"><title>103551754368</title>\n",
       "<polygon fill=\"none\" points=\"54.7148,-162.5 54.7148,-206.5 319.298,-206.5 319.298,-162.5 54.7148,-162.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"103.348\" y=\"-180.3\">flatten: Flatten</text>\n",
       "<polyline fill=\"none\" points=\"151.98,-162.5 151.98,-206.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"179.815\" y=\"-191.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"151.98,-184.5 207.649,-184.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"179.815\" y=\"-169.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"207.649,-162.5 207.649,-206.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"263.474\" y=\"-191.3\">(None, 7, 7, 512)</text>\n",
       "<polyline fill=\"none\" points=\"207.649,-184.5 319.298,-184.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"263.474\" y=\"-169.3\">(None, 25088)</text>\n",
       "</g>\n",
       "<!-- 103551811880&#45;&gt;103551754368 -->\n",
       "<g class=\"edge\" id=\"edge19\"><title>103551811880-&gt;103551754368</title>\n",
       "<path d=\"M187.006,-243.329C187.006,-235.183 187.006,-225.699 187.006,-216.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-216.729 187.006,-206.729 183.506,-216.729 190.506,-216.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103551604664 -->\n",
       "<g class=\"node\" id=\"node21\"><title>103551604664</title>\n",
       "<polygon fill=\"none\" points=\"72.2217,-81.5 72.2217,-125.5 301.791,-125.5 301.791,-81.5 72.2217,-81.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"110.348\" y=\"-99.3\">fc1: Dense</text>\n",
       "<polyline fill=\"none\" points=\"148.474,-81.5 148.474,-125.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"176.308\" y=\"-110.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"148.474,-103.5 204.143,-103.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"176.308\" y=\"-88.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"204.143,-81.5 204.143,-125.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"252.967\" y=\"-110.3\">(None, 25088)</text>\n",
       "<polyline fill=\"none\" points=\"204.143,-103.5 301.791,-103.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"252.967\" y=\"-88.3\">(None, 4096)</text>\n",
       "</g>\n",
       "<!-- 103551754368&#45;&gt;103551604664 -->\n",
       "<g class=\"edge\" id=\"edge20\"><title>103551754368-&gt;103551604664</title>\n",
       "<path d=\"M187.006,-162.329C187.006,-154.183 187.006,-144.699 187.006,-135.797\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-135.729 187.006,-125.729 183.506,-135.729 190.506,-135.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 103551387016 -->\n",
       "<g class=\"node\" id=\"node22\"><title>103551387016</title>\n",
       "<polygon fill=\"none\" points=\"75.7217,-0.5 75.7217,-44.5 298.291,-44.5 298.291,-0.5 75.7217,-0.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"113.848\" y=\"-18.3\">fc2: Dense</text>\n",
       "<polyline fill=\"none\" points=\"151.974,-0.5 151.974,-44.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"179.808\" y=\"-29.3\">input:</text>\n",
       "<polyline fill=\"none\" points=\"151.974,-22.5 207.643,-22.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"179.808\" y=\"-7.3\">output:</text>\n",
       "<polyline fill=\"none\" points=\"207.643,-0.5 207.643,-44.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"252.967\" y=\"-29.3\">(None, 4096)</text>\n",
       "<polyline fill=\"none\" points=\"207.643,-22.5 298.291,-22.5 \" stroke=\"black\"/>\n",
       "<text font-family=\"Times,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"252.967\" y=\"-7.3\">(None, 4096)</text>\n",
       "</g>\n",
       "<!-- 103551604664&#45;&gt;103551387016 -->\n",
       "<g class=\"edge\" id=\"edge21\"><title>103551604664-&gt;103551387016</title>\n",
       "<path d=\"M187.006,-81.3294C187.006,-73.1826 187.006,-63.6991 187.006,-54.7971\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"190.506,-54.729 187.006,-44.729 183.506,-54.729 190.506,-54.729\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SVG(model_to_dot(model_to_dot, show_shapes=True, show_layer_names=True).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_5 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 224, 224, 64)      1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 224, 224, 64)      36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 112, 112, 64)      0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 112, 112, 128)     147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 56, 56, 128)       0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 56, 56, 256)       590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 28, 28, 256)       0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 28, 28, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 14, 14, 512)       0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 7, 7, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 25088)             0         \n",
      "_________________________________________________________________\n",
      "fc1 (Dense)                  (None, 4096)              102764544 \n",
      "_________________________________________________________________\n",
      "fc2 (Dense)                  (None, 4096)              16781312  \n",
      "=================================================================\n",
      "Total params: 134,260,544\n",
      "Trainable params: 134,260,544\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = VGG16()\n",
    "# re-structure the model\n",
    "model.layers.pop()\n",
    "model = Model(inputs=model.inputs, outputs=model.layers[-1].output)\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SVG(model_to_dot(model, show_shapes=True, show_layer_names=True).create(prog='dot', format='svg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AttentionModel:\n",
    "    \n",
    "    def __init__(self):\n",
    "        \n",
    "        # load model\n",
    "        model = VGG16()\n",
    "        model.layers.pop()\n",
    "        # extract final 49x512 conv layer for context vectors\n",
    "        final_conv = Reshape([49,512])(model.layers[-4].output)\n",
    "        self.model = Model(inputs=model.inputs, outputs=final_conv)\n",
    "        print(self.model.summary())\n",
    "        \n",
    "        # model parameters\n",
    "        self.dim_ctx = 512\n",
    "        self.n_ctx = 49\n",
    "        self.lstm_cell_dim = 128\n",
    "        self.lstm_hidden_dim = 128\n",
    "        \n",
    "        # cell state MLP\n",
    "        self.c_mlp_hidden = 256\n",
    "        \n",
    "        self.inputs_c = Input(shape=(self.dim_ctx,))\n",
    "        f_c = Dense(self.c_mlp_hidden,activation=\"relu\")(self.inputs_c)\n",
    "        self.f_c = Dense(self.lstm_cell_dim,activation=None)(f_c)\n",
    "        \n",
    "        # hidden state MLP\n",
    "        self.h_mlp_hidden = 256\n",
    "        \n",
    "        self.inputs_h = Input(shape=(self.dim_ctx,))\n",
    "        f_h = Dense(self.h_mlp_hidden,activation=\"relu\")(self.inputs_h)\n",
    "        self.f_h = Dense(self.lstm_hidden_dim,activation=None)(f_h)\n",
    "        \n",
    "        # attention/alphas MLP\n",
    "        self.att_mlp_hidden = 256\n",
    "        \n",
    "        self.inputs_att = Input(shape=(self.dim_ctx+self.lstm_hidden_dim,))\n",
    "        x = Dense(self.att_mlp_hidden,activation=\"relu\")(self.inputs_att)\n",
    "        x = Dense(1,activation=None)(x)\n",
    "        self.alphas = Activation(\"softmax\")(x)\n",
    "        \n",
    "        self.sess = tf.Session()\n",
    "        \n",
    "    # Returns tensors for the initial cell_state and hidden_states\n",
    "    def init_lstm_states(self,contexts):\n",
    "        cell_state = self.sess.run(self.f_c,feed_dict={self.inputs_c:contexts})\n",
    "        hidden_state = self.sess.run(self.f_h,feed_dict={self.inputs_h:contexts})\n",
    "        return cell_state,hidden_state\n",
    "    \n",
    "    # Computes alpha values (attention weights) by passing context vectors + hidden state through MLP\n",
    "    # Includes hidden state by concatenating to end of alpha values\n",
    "    def generate_alphas(self,contexts,hidden_state):\n",
    "        batch_size = contexts.shape[0]\n",
    "        tiled_hidden_state = tf.tile([[hidden_state]],[batch_size,n_ctx,1])\n",
    "        concat_input = Concatenate(axis=-1)((contexts,tiled_hidden_state))\n",
    "        return self.sess.run(self.alphas,feed_dict={self.inputs_att:concat_input})\n",
    "\n",
    "    # Generates a soft-attention attention vector from alphas & context vectors\n",
    "    def get_soft_attention_vec(contexts,alphas):\n",
    "        return contexts*tf.reshape(alphas,[1,-1,1])\n",
    "    \n",
    "    # Generates VGG16 features from a batch of images\n",
    "    def get_features(images):\n",
    "        return self.sess.run(self.model.output,feed_dict={})\n",
    "        \n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Anaconda",
   "language": "python",
   "name": "anaconda"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
